'''
Created on 7 juin 2013

@author: Nils Amiet
'''

import threading
import time
import math

from twython import TwythonStreamer, Twython, TwythonRateLimitError
from twython.exceptions import TwythonError

from requests.exceptions import ConnectionError

from django.db import DatabaseError

from ITInfluence.models import Tweet, User
from ITInfluence.language import isEnglishTweet
from ITInfluence.hashtags import getSimilarHashtags

###
# Configuration starts here
###

'''
These values are provided for your personal use but since the code is distributed to multiple people,
and there can be only one machine using this account at the same time,
please use the values from your own Twitter account.
Register a regular Twitter account and then register an app at http://dev.twitter.com/apps
'''
APP_KEY = "pjoZEliZOQNl9D4tLGljA"
APP_SECRET = "1Wp2Jd2eiKVWYH8nJFm3wg6C3bnf10k1D25uzajloU"
OAUTH_TOKEN = "1479001058-kJibhcFtcHvUKvbFnKgUO8zPlTpb2MC6HCr1Z1z"
OAUTH_TOKEN_SECRET = "KmhIChKm9nENIXt5P5xfotOgxlaI9JfDBy3eZ4ZVKDc"


'''
Enables or disables the stream monitor.
The stream monitor periodically checks whether the incoming tweet rate is too low.
In such a case it restarts the stream with additional similar hashtags.
'''
isStreamMonitorEnabled = True

'''
Number of seconds between each stream monitor check.
'''
monitorCheckInterval = 300 # seconds

'''
Minimum number of incoming tweets per minute required.
Used by the stream monitor to decide whether it should restart with similar hashtags.
'''
incomingTweetRateThreshold = 0.5 # minimum tweets per minute

'''
Number of seconds after which the stream is stopped if not a single tweet was received.
REMARK: You should not set streamTimeout to a value that is less than monitorCheckInterval because
it may cause having two or more streams opened at the same time, which is forbidden
by the Twitter API.
'''
streamTimeout = 180 # seconds

###
# Configuration ends here
###

streamingHashtag = ""

class TwitterREST():
    '''Class to handle operations that require a call to the Twitter REST API'''
    
    TIME_INTERVAL_IN_SECONDS = (15 * 60) + 2 # add 2 seconds just to be sure
    MAX_FOLLOWERS_PER_REQUEST = 5000
    MAX_RETRY_COUNT = 30
    RETRY_INTERVAL_IN_SECONDS = 30
    isCollectingFollowers = False
    
    def __init__(self):
        self.twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
        
    def stopCollecting(self):
        self.isCollectingFollowers = False
        
    def sleepFor(self, seconds):
        startTime = time.time()
        for unused in range(seconds):
            time.sleep(1)
            if not self.isCollectingFollowers:
                return False
            
        # make sure we slept for long enough
        while True:
            currentTime = time.time()
            if currentTime - startTime < seconds:
                time.sleep(1)
            else:
                break
                
        return True
        
    def getFollowers(self, userId, getAll=False, retry=0):
        self.isCollectingFollowers = True
        
        try:
            totalFollowersCount = int(User.objects.get(id=userId).followers_count)
        except:
            totalFollowersCount = 0
        requiredCallsCount = int(math.ceil(totalFollowersCount/float(self.MAX_FOLLOWERS_PER_REQUEST)))
        partCount = 1
        
        # define rate limited operation
        def doGetFollowers(**kwargs):
            try:
                return self.twitter.get_followers_ids(user_id=userId, **kwargs)
            except ConnectionError as e:
                print(e)
                print("ConnectionError! Retrying...")
                self.sleepFor(10)
                return doGetFollowers(**kwargs)
        
        try:
            self.displayFollowersCollectionStatus(userId, partCount, requiredCallsCount)
            returnValue = doGetFollowers()
        except TwythonRateLimitError:
            # wait a complete Twitter time interval
            print("Sleeping until limit is lifted...")
            if not self.sleepFor(self.TIME_INTERVAL_IN_SECONDS):
                return # stop collecting
            # try again
            print("Retrying...")
            self.displayFollowersCollectionStatus(userId, partCount, requiredCallsCount)
            returnValue = doGetFollowers()
        except TwythonError as e:
            # User does not exist
            # User's account was probably deleted after we collected his tweets
            if "404" in str(e):
                print("WARNING: User %s does not exist!" % userId)
                return []
            if "401" in str(e):
                print("WARNING: User %s is probably private! Can't collect his/her followers." % userId)
                return []
            
            
            # Twitter is down / API changed / unknown error
            # wait a few seconds and retry
            print(e)
            print("Retrying in %s seconds..." % self.RETRY_INTERVAL_IN_SECONDS)
            if not self.sleepFor(self.RETRY_INTERVAL_IN_SECONDS):
                return
            if retry > self.MAX_RETRY_COUNT:
                print("Max retry count reached. Aborting...")
                return
            print("Retrying... attempt %s/%s" % (retry, self.MAX_RETRY_COUNT))
            return self.getFollowers(userId, getAll=getAll, retry=retry + 1)
        
        partCount += 1
        followers = returnValue["ids"]
        
        if getAll:
            
            while returnValue["next_cursor"] is not 0:
                try:
                    self.displayFollowersCollectionStatus(userId, partCount, requiredCallsCount)
                    returnValue = doGetFollowers(cursor=returnValue["next_cursor"])
                except TwythonRateLimitError:
                    # wait a complete Twitter time interval
                    print("Sleeping until limit is lifted...")
                    if not self.sleepFor(self.TIME_INTERVAL_IN_SECONDS):
                        return # stop collecting
                    # try again
                    print("Retrying...")
                    self.displayFollowersCollectionStatus(userId, partCount, requiredCallsCount)
                    returnValue = doGetFollowers(cursor=returnValue["next_cursor"])
                except TwythonError as e:
                    # Twitter is down / API changed / unknown error
                    # wait a few seconds and retry
                    print(e)
                    print("Retrying in %s seconds..." % self.RETRY_INTERVAL_IN_SECONDS)
                    if not self.sleepFor(self.RETRY_INTERVAL_IN_SECONDS):
                        return
                    if retry > self.MAX_RETRY_COUNT:
                        print("Max retry count reached. Aborting...")
                        return
                    print("Retrying... attempt %s/%s" % (retry, self.MAX_RETRY_COUNT))
                    return self.getFollowers(userId, getAll=getAll, retry=retry + 1)
                
                partCount += 1
                followers += returnValue["ids"]
        elif returnValue["next_cursor"] is not 0:
                print("Warning: you specified getAll=False but there was more than %s followers. Results are truncated!" % self.MAX_FOLLOWERS_PER_REQUEST)
        
        return followers
    
    def displayFollowersCollectionStatus(self, userId, partCount, requiredCallsCount):
        print("Getting followers for user %s (part %s/%s)..." % (userId, partCount, requiredCallsCount))
        
    def getUser(self, userId):
        # define rate limited operation
        def doGetUser():
            return self.twitter.show_user(user_id=userId)
        
        try:
            print("Getting details for user %s..." % userId)
            returnValue = doGetUser() 
        except TwythonRateLimitError:
            # wait a complete Twitter time interval
            print("Sleeping until limit is lifted...")
            self.sleepFor(self.TIME_INTERVAL_IN_SECONDS)
            # try again
            print("Retrying...")
            print("Getting details for user %s..." % userId)
            returnValue = doGetUser()
        
        print("done.")
        
        screen_name = returnValue["screen_name"]
        statuses_count = returnValue["statuses_count"]
        friends_count = returnValue["friends_count"]
        followers_count = returnValue["followers_count"]
        lang = returnValue["lang"]
        followersReady = True  # don't collect followers of followers
        user = User(id=userId, screen_name=screen_name, statuses_count=statuses_count, friends_count=friends_count, followers_count=followers_count, lang=lang, followers_ready=followersReady)
        
        return user

class TwitterStreaming(TwythonStreamer):
    '''Class to handle operations that deal with the Twitter Streaming API'''
    
    def setParams(self, stopTweetCount, stopTime):
        self.stopTweetCount = stopTweetCount
        self.stopTime = stopTime
        self.receivedTweetsCount = 0
        self.startTime = time.time()
        
        self.rateTweetCount = 0
        self.rateLastTime = time.time()
        
    def getRunningTime(self):
        currentTime = time.time()
        elapsedTime = currentTime - self.startTime # seconds
        elapsedTime /= 60.0 # convert to minutes
        return elapsedTime
    
    def getRate(self):
        '''Received tweets per minute'''
        receivedTweets = self.rateTweetCount
        timeInterval = (time.time() - self.rateLastTime) / 60.0 # minutes
        rate = receivedTweets / timeInterval
        
        self.rateLastTime = time.time()
        self.rateTweetCount = 0
        return rate
    
    def on_success(self, data):
        try:    
            text = data["text"]
            if not isEnglishTweet(text):
                print("Dropped tweet (considered non-english): " + text)
                return
            else:
                print("ENGLISH: " + text)
        except:
            pass
        
        try:
            # User
            author = data["user"]
            userId = author["id"]
            screen_name = author["screen_name"]
            statuses_count = author["statuses_count"]
            friends_count = author["friends_count"]
            followers_count = author["followers_count"]
            lang = author["lang"]
            followersReady = False
            userReady = True
            user = User(id=userId, screen_name=screen_name, statuses_count=statuses_count, friends_count=friends_count, followers_count=followers_count, lang=lang, followers_ready=followersReady, user_ready=userReady)
            
            if not User.objects.filter(id=userId).exists():
                try:
                    user.save()
                except:
                    print("ERROR: failed saving user")
            
            # Tweet
            tweetId = data["id_str"]
            in_reply_to_status_id = data["in_reply_to_status_id"]
            if in_reply_to_status_id is None:
                in_reply_to_status_id = 0
                
            hashtags = ""
            try:
                ht = data["entities"]["hashtags"]
                tags = [x["text"] for x in ht]
                hashtags = " ".join(tags)
            except:
                # this tweet doesn't contain any hashtag
                pass
                
            text = data["text"]
            created_at = data["created_at"]
            timestamp = time.strftime('%Y-%m-%d %H:%M:%S+00:00', time.strptime(created_at, '%a %b %d %H:%M:%S +0000 %Y'))
            polarity = 0
            polarityReady = False
            
            tweet = Tweet(id=tweetId, user=user, in_reply_to_status_id=in_reply_to_status_id, text=text, created_at=timestamp, polarity=polarity, polarity_ready=polarityReady, hashtags=hashtags)
            tweet.save()

            # update received tweets count
            self.receivedTweetsCount += 1
            self.rateTweetCount += 1
            
            # check stop conditions
            if (self.receivedTweetsCount is not 0 and self.receivedTweetsCount >= self.stopTweetCount) or (self.stopTime is not 0 and self.getRunningTime() >= self.stopTime):
                self.disconnect()
        except KeyError:
            '''Some tweets are badly sent and are missing user or text'''
            pass
        except DatabaseError:
            '''Bad response (invalid JSON)'''
            pass
        except Exception as e:
            print(e)
            pass

    def on_error(self, status_code, data):
        print(status_code, data)





'''FUNCTIONS'''
def readStream(filterHashtag, stopTweetCount, stopTime, stream):
    stream.setParams(stopTweetCount, stopTime)
    
    try:
        stream.statuses.filter(track=filterHashtag)
        print("Stopped streaming")
    except Exception as e:
        print(e)
        stream.disconnect()
        print("Stopped streaming (timeout)")
        
        updateHashtagsUsingSimilar(stream) # retry with similar hashtags
    
def launchStreamMonitor():
    global stream
    t = threading.Thread(target=monitorStream, args=(stream,))
    t.daemon = True
    print("Launched stream monitor...")
    t.start()
    
def monitorStream(stream):
    # monitor agent that checks incoming tweet rate
    # relaunch stream with similar hashtags if rate is too low
    
    while True:
        try:
            if stream.connected:
                print("Stream is connected.")
                tweetsPerMinute = stream.getRate()
                print("Current rate: %s tweets/min" % tweetsPerMinute)
                
                streamStartInterval = time.time() - stream.startTime
                
                if tweetsPerMinute < incomingTweetRateThreshold and streamStartInterval >= streamTimeout:
                    print("Not receiving enough tweets matching this set of hashtags.")
                    print("Retrying with additional similar hashtags...")
                    updateHashtagsUsingSimilar(stream)
                
            else:
                print("Stream is disconnected.")
        except Exception as e:
            print(e)
            print("WARNING: stream not instantiated. Stopping monitor...")
            break
        
        # sleep until next check
        time.sleep(monitorCheckInterval)
        
def updateHashtagsUsingSimilar(stream):
    currentTags =  [tag.replace("#", "") for tag in streamingHashtag.split(",")]
    similarTags = []
    for tag in currentTags:
        similar = getSimilarHashtags(tag)
        similar = sorted(similar, key=lambda x: x[1], reverse=True)
        similarCount = 2
        similar = similar[:similarCount] # N most similar tags
        similar = [tag for tag, unused in similar]
        
        similarTags += similar
    similarTags += currentTags
    similarTags = set(similarTags) # remove duplicates
        
    newFilterTags = ",".join(["#" + tag for tag in similarTags]) # convert to comma seperated string with leading # for each tag
    print("Current tags: %s" % streamingHashtag)
    print("New tags: %s" % newFilterTags)
    
    # stop stream
    stream.disconnect()
    
    # restart with new tags
    stopTweetCount = stream.stopTweetCount - stream.receivedTweetsCount
    stopTime = stream.stopTime
    del stream
    
    startStreaming(newFilterTags, stopTweetCount, stopTime)

def startStreaming(filterHashtag="twitter", stopTweetCount=0, stopTime=0):
    global streamingHashtag
    streamingHashtag = filterHashtag
    
    global stream
    
    # start streaming
    t = threading.Thread(target=readStream, args=(filterHashtag, stopTweetCount, stopTime, stream))
    t.daemon = True
    print("started reading stream...")
    t.start()
    
def toggleStreaming():
    global stream
    
    if not stream.connected:
        startStreaming()
    else:
        print("disconnecting stream...")
        stream.disconnect()
    
# instantiate stream
stream = TwitterStreaming(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET, timeout=streamTimeout)
# monitor stream
if isStreamMonitorEnabled:
    launchStreamMonitor()
